import matplotlib.pyplot as plt
import pandas as pd

pd.set_option('display.max_columns',20) #给最大列设置为10列
pd.set_option('display.max_rows',100)#设置最大可见100行

train = pd.read_csv(r'C:\Users\Aiomi\Desktop\nlp\train.csv')
test = pd.read_csv(r'C:\Users\Aiomi\Desktop\nlp\test.csv')

# print(trian.info)
# print(trian.describe())
print(train.head())

temp = train.groupby('sentiment').count()['text'].reset_index().sort_values(by='text',ascending=False)
temp.style.background_gradient(cmap = 'Purples')

import  matplotlib.pyplot as plt
import  seaborn as sns
#
# plt.figure(figsize=(12,6))
# sns.countplot(x="sentiment",data = train)
# plt.show()



train['Num_words_ST']=train['selected_text'].apply(lambda x:len(str(x).split()))
train['Num_word_text']=train['text'].apply(lambda x:len(str(x).split()))
train['diffence_in_words'] = train['Num_word_text']-train['Num_words_ST']

plt.Figure(figsize=(12,6))
pl = sns.kdeplot(train['Num_words_ST'],fill=True,color='r')
pl = sns.kdeplot(train['Num_word_text'],fill=True,color='r')
#plt.show()

k = train[train['Num_word_text']<=2]
#print(k[k['sentiment']=='positive'])

import re
import string

def clean_text(text):
    '''Make text lowercase, remove text in square brackets,remove links,remove punctuation
    and remove words containing numbers.'''
    text = str(text).lower()
    text = re.sub('\[.*?\]', '', text)
    text = re.sub('https?://\S+|www\.\S+', '', text)
    text = re.sub('<.*?>+', '', text)
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    text = re.sub('\n', '', text)
    text = re.sub('\w*\d\w*', '', text)
    return text

train['text'] = train['text'].apply(lambda x:clean_text(x))
train['selected_text'] = train['selected_text'].apply(lambda x:clean_text(x))

from collections import Counter
train['temp_list'] = train['selected_text'].apply(lambda x:str(x).split())
top = Counter([item for sublist in train['temp_list']for item in sublist])
temp = pd.DataFrame(top.most_common(20))
temp.columns = ['Common_words','count']
#print(temp)

Positive_sent = train[train['sentiment']=='positive']
Negative_sent = train[train['sentiment']=='negative']
Neutral_sent = train[train['sentiment']=='neutral']

top_positive = Counter([item for sublist in Positive_sent["filter_list"] for item in sublist])
filter_posotive = pd.DataFrame(top.most_common(20))
filter_posotive.columns = ["Common_words", "count"]
filter_posotive.style.background_gradient(cmap="Greens")

top_negative = Counter([item for sublist in Negative_sent["filter_list"] for item in sublist])
filter_negative = pd.DataFrame(top_negative.most_common(20))
filter_negative.columns = ["Common_words", "count"]
filter_negative.style.background_gradient(cmap="Reds")

train["label"] = train.sentiment.map({"negative": 0, "positive": 1, "neutral": 2})
train.head()